package com.kinth.work.webmagic.huicong;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 
 * <p>
 * Title: 慧聪网企业数据抓取
 * </p>
 * <p>
 * Description:
 * http://cfqj.b2b.hc360.com/，这是具体的公司页面。（暂时没有发现其他类型的页面）。提取相关item.目前提取了
 * 公司名称，产品，地址，qq,联系方式，公司简介。微信是提取不出来的。因为它是直接跳出一个界面。 5. 由于程序太大。还没有系统测试过。单元测试完成。
 * </p>
 * 
 * @author kaikai
 * @2017年3月21日
 */

public class GetInfo implements PageProcessor, Runnable
{
	public static final int NUM_ONCE = 10;// 每次从数据库里面取出的值
	private static String companyUrlPath = System.getProperty("user.dir") + File.separator + "company.txt";
	private static File companyUrlFile = new File(companyUrlPath);
	// private String industry;
	// private String urlNow;
	private Site site = Site.me()
			.setUserAgent("Mozilla/5.0 (X11; U; Linux i686; zh-CN; rv:1.9.1.2) " + "Gecko/20090803").setRetryTimes(3)
			.setSleepTime(100);

	/**
	 * 1. 提取相关的item
	 * 
	 * @param page
	 */
	@Override
	public void process(Page page)
	{
		// 公司名称
		String name = page.getHtml().xpath("//body/div[@id='innerBgBox']"
				+ "/div[@id='mainbox']//div[@class='columnbox colrel']" + "//div[@class='comInfogo']/ul/li[1]/a/text()")
				.toString();

		String people = null;
		String address = null;
		String phone1 = null;
		String phone2 = null;
		String phone3 = null;
		String qq = null;
		String introduction = null;
		String product = null;
		if (name != null)
		{
			// 产品
			product = page.getHtml().xpath("//head/title/text()").toString();

			// 联系人
			people = page.getHtml()
					.xpath("//body/div[@id='innerBgBox']" + "/div[@id='mainbox']//div[@class='columnbox colrel']"
							+ "//div[@class='comInfogo']/ul/li[2]/span/a[1]/text()")
					.toString();

			// 地址
			address = page.getHtml()
					.xpath("//body/div[@id='innerBgBox']" + "/div[@id='mainbox']//div[@class='columnbox colrel']"
							+ "//div[@class='archives colrel']/div[3]/text()")
					.toString();

			// 联系qq
			qq = page.getHtml().xpath("//body/div[@id='innerBgBox']"
					+ "/div[@id='mainbox']//div[@class='columnbox colrel']" + "//div[@class='comInfogo']//li[2]/a")
					.links().toString();

			// 电话
			String phoneTemp = page.getHtml()
					.xpath("//body/div[@id='innerBgBox']/div[@id='mainbox']//div[@class='txtb']"
							+ "//div[@class='content-inner']/div[2]/span/text()")
					.toString();
			phone1 = page.getHtml().xpath("//body/div[@id='innerBgBox']/div[@id='mainbox']//div[@class='txtb']"
					+ "//div[@class='content-inner']/div[2]/em/text()").toString();
			if (phone1 != null)
			{
				phone1 = phoneTemp + phone1;
			}

			// 手机
			phoneTemp = page.getHtml().xpath("//body/div[@id='innerBgBox']/div[@id='mainbox']//div[@class='txtb']"
					+ "//div[@class='content-inner']/div[3]/span/text()").toString();
			phone2 = page.getHtml().xpath("//body/div[@id='innerBgBox']/div[@id='mainbox']//div[@class='txtb']"
					+ "//div[@class='content-inner']/div[3]/em/text()").toString();
			if (phone2 != null)
			{
				phone2 = phoneTemp + phone2;
			}

			// 传真
			phoneTemp = page.getHtml().xpath("//body/div[@id='innerBgBox']/div[@id='mainbox']//div[@class='txtb']"
					+ "//div[@class='content-inner']/div[4]/span/text()").toString();
			phone3 = page.getHtml().xpath("//body/div[@id='innerBgBox']/div[@id='mainbox']//div[@class='txtb']"
					+ "//div[@class='content-inner']/div[4]/em/text()").toString();
			if (phone3 != null)
			{
				phone3 = phoneTemp + phone3;
			}

			// 公司介绍
			introduction = page.getHtml().xpath("//body/div[@id='innerBgBox']" + "/div[@id='mainbox']"
					+ "//div[@class='contentbox']" + "//div[@class='cAbout']/text()").toString();

			// 此处是调用phantomJS来提取动态加载的qq。但是因为目前慧聪网此处加载的qq是网站的客服，就是全部都一样。所以先
			// 注释掉整个if
			// if(qq == null)
			// {
			// //companyServiceMod
			// try
			// {
			// qq = JSUtil.getParseredHtml2(urlNow);
			// //System.out.println("测试： "+qq);
			// String regex = "<a[\\s\\S]*?>";
			// Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
			// Matcher ma = pa.matcher(qq);
			// if(ma.find())
			// {
			// qq = ma.group(0);
			// }
			//
			// } catch (IOException e)
			// {
			// e.printStackTrace();
			// }
			//
			// }
			if (qq != null)
			{
				String[] tempqq = qq.split("&");
				if (tempqq.length >= 2)
				{
					String tempqq2 = tempqq[1];

					tempqq = tempqq2.split("=");
					if (tempqq.length >= 2)
					{
						tempqq2 = tempqq[1];

						qq = tempqq2;
					} else
					{
						qq = null;
					}

				} else
				{
					qq = null;
				}
			}

			// 试着industry从数据库里面取出来
			// System.out.println("GetInfo: url"+page.getUrl());
			// 只用取一次即可
			String urlNow = page.getUrl().toString();
			ArrayList<String> industryList = dbUtil.queryIndustry("huicong_url3", urlNow);
			String industry = null;
			if (industryList.size() > 0)
			{
				industry = industryList.get(0);
			}
			page.putField("name", name);
			page.putField("product", product);
			page.putField("address", address);
			page.putField("people", people);
			page.putField("qq", qq);
			page.putField("introduction", introduction);
			page.putField("phone1", phone1);
			page.putField("phone2", phone2);
			page.putField("phone3", phone3);
			page.putField("industry", industry);
			page.putField("url", urlNow);
		} else
		{
			name = page.getHtml()
					.xpath("//body/div[@class='newWrap']" + "//div[@class='proBox25left']/div[@class='columnbox']"
							+ "/div[@class='leftBoxCon qyArchives']/h2/a/text()")
					.toString();
			System.out.println("name2: " + name);
			if (name != null)
			{
				// 联系人
				people = page.getHtml()
						.xpath("//body/div[@class='newWrap']" + "//div[@class='proBox25left']/div[@class='columnbox']"
								+ "/div[@class='leftBoxCon qyArchives']/div[@class='ArchivesList']" + "//p/a/text()")
						.toString();

				// 地址
				address = page.getHtml()
						.xpath("//body/div[@class='newWrap']" + "//div[@class='proBox25left']/div[@class='columnbox']"
								+ "/div[@class='leftBoxCon qyArchives']/div[@class='ArchivesList']"
								+ "/ul/li[3]/div/text()")
						.toString();

				qq = page.getHtml()
						.xpath("//body/div[@class='newWrap']" + "//div[@class='proBox25left']/div[@class='columnbox']"
								+ "/div[@class='leftBoxCon qyArchives']/div[@class='ArchivesList']"
								+ "//div[@class='aRight']/a")
						.links().toString();

				// 此处是调用phantomJS来提取动态加载的qq。但是因为目前慧聪网此处加载的qq是网站的客服，就是全部都一样。所以先
				// 注释掉整个if
				// if(qq == null)
				// {
				// try
				// {
				// qq = JSUtil.getParseredHtml2(urlNow);
				// String regex = "<a[\\s\\S]*?>";
				// Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
				// Matcher ma = pa.matcher(qq);
				// if(ma.find())
				// {
				// qq = ma.group(0);
				// }
				//
				// } catch (IOException e)
				// {
				// e.printStackTrace();
				// }
				//
				// }
				if (qq != null)
				{
					String[] tempqq = qq.split("&");
					if (tempqq.length >= 2)
					{
						String tempqq2 = tempqq[1];

						tempqq = tempqq2.split("=");
						if (tempqq.length >= 2)
						{
							tempqq2 = tempqq[1];

							qq = tempqq2;
						} else
						{
							qq = null;
						}

					} else
					{
						qq = null;
					}

				}

				String phoneTemp = page.getHtml()
						.xpath("//body/div[@class='newWrap']"
								+ "//div[@class='proBox25left']//div[@class='leftBoxCon ContactBox']"
								+ "/ul/li[2]/span[1]/text()")
						.toString();
				phone1 = page.getHtml()
						.xpath("//body/div[@class='newWrap']"
								+ "//div[@class='proBox25left']//div[@class='leftBoxCon ContactBox']"
								+ "/ul/li[2]/span[2]/text()")
						.toString();
				phone1 = phoneTemp + phone1;

				phoneTemp = page.getHtml()
						.xpath("//body/div[@class='newWrap']"
								+ "//div[@class='proBox25left']//div[@class='leftBoxCon ContactBox']"
								+ "/ul/li[3]/span[1]/text()")
						.toString();
				phone2 = page.getHtml()
						.xpath("//body/div[@class='newWrap']"
								+ "//div[@class='proBox25left']//div[@class='leftBoxCon ContactBox']"
								+ "/ul/li[3]/span[2]/text()")
						.toString();
				phone2 = phoneTemp + phone2;

				phoneTemp = page.getHtml()
						.xpath("//body/div[@class='newWrap']"
								+ "//div[@class='proBox25left']//div[@class='leftBoxCon ContactBox']"
								+ "/ul/li[4]/span[1]/text()")
						.toString();
				phone3 = page.getHtml()
						.xpath("//body/div[@class='newWrap']"
								+ "//div[@class='proBox25left']//div[@class='leftBoxCon ContactBox']"
								+ "/ul/li[4]/span[2]/text()")
						.toString();
				if (phone3 != null)
				{
					phone3 = phoneTemp + phone3;
				}

				introduction = page.getHtml()
						.xpath("//body/div[@class='newWrap']" + "//div[@class='proBox75Rig']/div[@class='columnbox']"
								+ "//div[@class='conpanyBoxSingle']/p/text()")
						.toString();
				if (introduction == null)
				{
					introduction = page.getHtml()
							.xpath("//body/div[@class='newWrap']"
									+ "//div[@class='proBox75Rig']/div[@class='columnbox']"
									+ "//div[@class='conpanyBox']/p/text()")
							.toString();
				}
				product = page.getHtml().xpath("//head/meta[@name='description']").toString();
				// 只用取一次即可
				String urlNow = page.getUrl().toString();
				ArrayList<String> industryList = dbUtil.queryIndustry("huicong_url3", urlNow);
				String industry = null;
				if (industryList.size() > 0)
				{
					industry = industryList.get(0);
				}
				page.putField("name", name);
				page.putField("product", product);
				page.putField("address", address);
				page.putField("people", people);
				page.putField("qq", qq);
				page.putField("introduction", introduction);
				page.putField("phone1", phone1);
				page.putField("phone2", phone2);
				page.putField("phone3", phone3);
				page.putField("industry", industry);
				page.putField("url", urlNow);
			}
		}

	}

	@Override
	public Site getSite()
	{
		return site;
	}

	// 一个代替main函数的函数
	public static void runSpider()
	{
		System.out.println("爬取具体信息开始");
		// 从记录文件里面取出记录
		int rank = FileUtil.getCompanyUrl2();
		// 取出，每次10个
		ArrayList<UrlHuiCong> urlList = dbUtil.queryUrl("huicong_url3", rank, NUM_ONCE);
		GetInfo getInfo = new GetInfo();
		Spider spider = Spider.create(getInfo);
		while (true)
		{
			System.out.println("新一次爬取具体信息开始");
			if (urlList.size() == 0)
			{
				try
				{
					Thread.sleep(1000 * 10);
					// 提醒数据库为空，以便完成爬取之后暂停
					System.out.println("GetInfo: 当下公司链接已经爬取完毕。请结合其他列表情况判断是否已经完成");
				} catch (InterruptedException e)
				{
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			} else
			{
				for (UrlHuiCong urlHuiCong : urlList)
				{

					String startUrlTemp = urlHuiCong.getUrl();

					spider.addUrl(startUrlTemp).addPipeline(new DataPipeline()).run();
				}
			}

			int nextRank = FileUtil.getCompanyUrl2() + urlList.size();
			FileUtil.writeCompanyUrl2(nextRank);
			urlList = dbUtil.queryUrl("huicong_url3", nextRank, NUM_ONCE);
		}
	}

	public static void main(String[] args)
	{
		GetInfo getInfo = new GetInfo();

		// http://huisheng99.b2b.hc360.com/
		// http://hui88889999.b2b.hc360.com/
		// http://liyue02.b2b.hc360.com/
		// http://jzjymy999.b2b.hc360.com/
		// http://1356141042.b2b.hc360.com/
		// http://shhengda.b2b.hc360.com/
		// http://bjxlf.b2b.hc360.com/ 忘了说，这种因为没有qq，所以没有抓。
		//http://05702930.b2b.hc360.com
		String startUrlTemp = "http://05702930.b2b.hc360.com";

		Spider.create(getInfo).addUrl(startUrlTemp)
				.addPipeline(new DataPipeline())
				.run();
	}

	@Override
	public void run()
	{
		runSpider();

	}
}